support for image attachments when classifying questions

Author: escattone

kitsune/llm/questions/classifiers.py

@@ -32,6 +32,7 @@ def classify_question(question: "Question") -> dict[str, Any]:
     payload: dict[str, Any] = {
         "subject": question.title,
         "question": question.content,
+        "image_urls": [image.get_absolute_url() for image in question.get_images()],
         "product": product,
         "topics": get_taxonomy(
             product, include_metadata=["description", "examples"], output_format="JSON"

kitsune/llm/questions/prompt.py

@@ -1,5 +1,9 @@
+from typing import Any
+
 from langchain.output_parsers import ResponseSchema, StructuredOutputParser
-from langchain.prompts import ChatPromptTemplate
+from langchain.prompts import ChatPromptTemplate, MessagesPlaceholder
+from langchain.schema import HumanMessage
+from langchain.schema.runnable import RunnableLambda
 
 SPAM_INSTRUCTIONS = """
 # Role and goal
@@ -120,10 +124,10 @@
 )
 
 
-spam_prompt = ChatPromptTemplate(
+spam_prompt_with_human_message = ChatPromptTemplate(
     (
         ("system", SPAM_INSTRUCTIONS),
-        ("human", USER_QUESTION),
+        MessagesPlaceholder("human_message"),
     )
 ).partial(format_instructions=spam_parser.get_format_instructions())
 
@@ -134,3 +138,23 @@
         ("human", USER_QUESTION),
     )
 ).partial(format_instructions=topic_parser.get_format_instructions())
+
+
+def add_human_message(inputs: dict) -> dict:
+    """
+    Adds the human message to the inputs dict and returns it. Ensures
+    that the human message includes image URL's if they're present.
+    """
+    image_urls = inputs.pop("image_urls", None)
+
+    content: list[dict[str, Any]] = [dict(type="text", text=USER_QUESTION.format(**inputs))]
+
+    if image_urls:
+        for image_url in image_urls:
+            content.append(dict(type="image_url", image_url=dict(url=image_url)))
+
+    inputs.update(human_message=[HumanMessage(content=content)])
+    return inputs
+
+
+spam_prompt = RunnableLambda(add_human_message) | spam_prompt_with_human_message